﻿//DbfSource.cs
//Copyright (c) 2013 StockSharp LLC, all rights reserved.
//This code module is part of StockSharp library.
//This code is licensed under the GNU GENERAL PUBLIC LICENSE Version 3.
//See the file License.txt for the license details.
//More info on: http://stocksharp.com

namespace StockSharp.Algo.History.Rts
{
	using System;
	using System.Collections.Generic;
	using System.IO;
	using System.Linq;

	using DotNetDBF;

	using Ecng.Collections;
	using Ecng.Common;

	using StockSharp.Logging;
	using StockSharp.BusinessEntities;

	using LumiSoft.Net.FTP.Client;

	class DbfSource : FileSource
	{
		private DateTime _lastListDate;
		private string[] _dbfRemoteItems;
		private readonly ExchangeBoard _exchangeBoard;

		public DbfSource(ISecurityStorage securityStorage, SecurityIdGenerator securityIdGenerator)
			: base(securityStorage, securityIdGenerator)
		{
			_exchangeBoard = new ExchangeBoard();
			_exchangeBoard.ApplyDefaultHolidays(new DateTime(2001, 1, 1), LoggingHelper.Now);
		}

		public bool SaveRtsStdCombinedOnly { get; set; }

		protected override IDictionary<Security, List<Trade>> OnGetTrades(FTP_Client client, DateTime date)
		{
			if (_lastListDate.Date != DateTime.Today)
			{
				_dbfRemoteItems = client
					.GetList(FtpDirectory)
					.Select(item => item.Name)
					.ToArray();

				_lastListDate = DateTime.Today;
			}

			var trades = new Dictionary<Security, List<Trade>>();

			var localPath = DownloadDbf(client, date);

			if (null != localPath)
			{
				ProcessDbf(localPath, trades, date);
				CombineRtsStd(trades, date);
			}

			return trades;
		}

		private string DownloadDbf(FTP_Client client, DateTime time)
		{
			var dirName = "{0}{1:00}{2:00}".Put(time.Year, time.Month, time.Day); // 20080101
			var zipName = dirName + ".zip"; // 20080101.zip

			var exactZipName = _dbfRemoteItems.FirstOrDefault(i => i.CompareIgnoreCase(zipName));

			if (exactZipName == null && !_dbfRemoteItems.Contains(dirName))
				return null;

			var localZip = Path.Combine(DumpFolder, zipName); // <dump>\20080101.zip
			var localPath = Path.Combine(DumpFolder, dirName); // <dump>\20080101

			if (!File.Exists(localZip))
			{
				// если зип лежит в корне на уровне директорий с датами (УкрБиржа)
				if (exactZipName != null)
				{
					client
						.DownloadFile(FtpDirectory, exactZipName, localZip)
						.Extract(localPath);
				}
				else
				{
					var remoteDir = Path.Combine(FtpDirectory, dirName);

					var remoteFiles = client.GetList(remoteDir);
					var dbfRemoteFiles = remoteFiles.Where(file => file.Name.ContainsIgnoreCase(".dbf")).ToArray();
					var zipRemoteFile = remoteFiles.FirstOrDefault(item => item.Name.CompareIgnoreCase(zipName));

					if (zipRemoteFile != null) // есть зип на фтп
					{
						// локально нет какого-то DBF - качаем и распаковываем зип
						if (dbfRemoteFiles.Any(file => !File.Exists(Path.Combine(localPath, file.Name))))
						{
							client
								.DownloadFile(remoteDir, zipRemoteFile.Name, localZip)
								.Extract(localPath);
						}
					}
					else // нет зип на фтп
					{
						if (!Directory.Exists(localPath))
							Directory.CreateDirectory(localPath);

						dbfRemoteFiles.ForEach(file =>
						{
							var localFile = Path.Combine(localPath, file.Name);

							if (!File.Exists(localFile))
								client.DownloadFile(remoteDir, file.Name, localFile);
						});
					}
				}
			}
			else
			{
				// если зип был скачан, но не распакован
				if (!Directory.Exists(localPath))
				{
					localZip.Extract(localPath);
				}
			}

			return localPath;
		}

		private void ProcessDbf(string localPath, IDictionary<Security, List<Trade>> trades, DateTime date)
		{
			var securityByName = new Dictionary<string, Security>();

			foreach (var fileName in new[] { "f07.dbf", "o07.dbf" }) // фьючерсы идут вначале, dayXX.dbf игнорируем
			{
				var file = Path.Combine(localPath, fileName);

				var isDayClearing = false;

				if (!File.Exists(file))
				{
					file = Path.Combine(localPath, "day" + fileName);

					// если нет данных ни за вечерний клиринг, ни за дневнной (day), значит торгов не было
					if (!File.Exists(file))
						continue;

					isDayClearing = true;
				}

				//if (fileName == null)
				//	throw new InvalidOperationException();

				var isFuture = !fileName.Contains("o");

				using (var reader = new DBFReader(file))
				{
					var nameIndex = reader.GetFieldIndex("CONTRACT");
					var codeIndex = reader.GetFieldIndex("NAME");
					var minStepSizeIndex = reader.GetFieldIndex("TICK");
					var minStepPriceIndex = reader.GetFieldIndex("TICK_PRICE");
					var minLotSizeIndex = isFuture ? reader.GetFieldIndex("LOT_VOLUME") : -1;
					var expirationIndex = reader.GetFieldIndex("EXECUTION");
					var isSpotIndex = isFuture ? reader.GetFieldIndex("SPOT") : -1; // базовый спот для T+N
					var baseIndex = isFuture ? reader.GetFieldIndex("BASE") : -1;
					var baseFutIndex = isFuture ? reader.GetFieldIndex("BASE_FUT") : -1; // базовый инструмент фьючерса
					var futContrIndex = !isFuture ? reader.GetFieldIndex("FUT_CONTR") : -1; // базовый фьючерс опциона
					var multiLegIndex = isFuture ? reader.GetFieldIndex("MULTILEG") : -1;

					for (var i = 0; i < reader.RecordCount; i++)
					{
						var values = reader.NextRecord();

						if (values[minStepSizeIndex] == null)
							continue;

						if (multiLegIndex > -1 && (decimal)values[multiLegIndex] == 1)
							continue;

						var secCode = ((string)values[codeIndex]).Trim();

						var secName = ((string)values[nameIndex]).Trim();
						var secExpiration = ((string)values[expirationIndex]).ToDateTime("yyyy\\/MM\\/dd");

						var underlyingId = "";
						var secType = isFuture ? SecurityTypes.Future : SecurityTypes.Option;

						var saveSecurity = true;

						if (isFuture)
						{
							var isSpotValue = isSpotIndex > -1 ? ((string)values[isSpotIndex]).Trim() : "";
							if (baseIndex > -1 && isSpotValue == "SPOT")
							{
								var baseValue = baseIndex > -1 ? ((string)values[baseIndex]).Trim() : "";

								underlyingId = SecurityIdGenerator.GenerateId(baseValue, null, ExchangeBoard);
								//underlyingId = baseValue + "@RTS"; // RTS-Std
								secType = SecurityTypes.Stock;
								secCode = secName; // для уникальности GAZP18->GAZP-18.01.01

								saveSecurity = !SaveRtsStdCombinedOnly; // не сохранять несклееные RTS STD

								// споты на VTBR 05.10.2009 уменьшили в 10 раз шаг цены и пункт
								// до этого они были 0.0001 и 0.1 соответственно
								if (underlyingId == "VTBR@RTS")
								{
									values[minStepSizeIndex] = 0.00001m;
									values[minStepPriceIndex] = 0.01m;
								}
							}
							else if (baseFutIndex > -1)
							{
								var baseValue = ((string)values[baseFutIndex]).Trim();
								underlyingId = SecurityIdGenerator.GenerateId(baseValue, null, ExchangeBoard);
							}
						}
						else
						{
							// option
							if (futContrIndex > -1)
							{
								var futContr = ((string)values[futContrIndex]).Trim();
								underlyingId = securityByName[futContr].Id;
							}
						}

						var secId = SecurityIdGenerator.GenerateId(secCode, null, ExchangeBoard);
						var security = SecurityStorage.LoadBy("Id", secId);

						var minLotSize = isFuture ? values[minLotSizeIndex].To<int>() : 1;
						var minStepPrice = values[minStepPriceIndex].To<decimal>().RemoveTrailingZeros();
						var minStepSize = values[minStepSizeIndex].To<decimal>().RemoveTrailingZeros();

						if (security == null)
						{
							security = new Security
							{
								Id = secId,
								Code = secCode,
								Name = secName,
								UnderlyingSecurityId = underlyingId,
								Type = secType,
								Class = string.Empty,
								ExpiryDate = secExpiration,
								MinLotSize = minLotSize,
								MinStepPrice = minStepPrice,
								MinStepSize = minStepSize,
								ExtensionInfo = new Dictionary<object, object>(),
								ExchangeBoard = ExchangeBoard,
							};

							if (saveSecurity)
								SecurityStorage.Save(security);
						}
						else
						{
							// инструмент был создан через FinamHistorySource
							// или изменился шаг цены
							if (security.MinStepSize == 0 || security.MinStepSize != minStepSize)
							{
								security.Type = secType;
								security.UnderlyingSecurityId = underlyingId;
								security.ExpiryDate = secExpiration;
								security.MinLotSize = minLotSize;
								security.MinStepPrice = minStepPrice;
								security.MinStepSize = minStepSize;

								if (saveSecurity)
								{
									SecurityStorage.Save(security);
								}
							}

							if (!isDayClearing)
							{
								// обновить пункт
								security.MinStepPrice = values[minStepPriceIndex].To<decimal>().RemoveTrailingZeros();	
							}
						}

						securityByName[secName] = security;
					}
				}
			}

			if (securityByName.Count == 0)
				return;

			// 28.12.2007 есть сделки по инструменту GMKR-3.08_120308PA 55000, но нет его описания
			if (date == new DateTime(2007, 12, 28) && ExchangeBoard == ExchangeBoard.Forts)
			{
				var security = SecurityStorage.LoadBy("Id", "GM55000O8@RTS");

				if (security == null ||
					security.MinStepSize == 0) // инструмент был создан через FinamHistorySource
				{
					if (security == null)
					{
						security = new Security
						{
							Id = "GM55000O8@RTS",
							Code = "GM55000O8",
							Name = "GMKR-3.08_120308PA 55000",
							Class = string.Empty,
							ExtensionInfo = new Dictionary<object, object>(),
							ExchangeBoard = ExchangeBoard,
						};
					}
					
					security.Type = SecurityTypes.Option;
					security.UnderlyingSecurityId = "GMH8@RTS";
					security.ExpiryDate = new DateTime(2008, 03, 12);
					security.MinLotSize = 1;
					security.MinStepPrice = 1;
					security.MinStepSize = 1;

					SecurityStorage.Save(security);
				}

				securityByName.Add(security.Name, security);
			}

			foreach (var file in Directory.GetFiles(localPath, "*deal.dbf"))
			{
				if (file.Contains("multileg"))
					continue;

				using (var reader = new DBFReader(file))
				{
					var nameIndex = reader.GetFieldIndex("ISIN");

					var idIndex = reader.GetFieldIndex("ID_DEAL");
					var volumeIndex = reader.GetFieldIndex("VOL");
					var priceIndex = reader.GetFieldIndex("PRICE");
					var dateIndex = reader.GetFieldIndex("DATE");
					var timeIndex = reader.GetFieldIndex("TIME");

					var typeIndex = reader.GetFieldIndex("TYPE");

					for (var i = 0; i < reader.RecordCount; i++)
					{
						var values = reader.NextRecord();

						var price = values[priceIndex].To<decimal>();

						// http://forum.rts.ru/viewtopic.asp?t=19730
						if (price == 0)
							continue;

						var dateTime = (DateTime)values[dateIndex];
						dateTime = dateTime.Add(values[timeIndex].To<DateTime>().TimeOfDay);

						var id = values[idIndex].To<long>();

						if (id <= 0)
							throw new InvalidOperationException("Неправильный идентификатор сделки {0}.".Put(id));

						var secName = ((string)values[nameIndex]).Trim();

						var security = securityByName.TryGetValue(secName);

						if (security == null)
							throw new InvalidOperationException("Инструмент с именем {0} не найден для сделки {1}.".Put(secName, id));

						var isSystem = values[typeIndex].To<int>() == 0;

						if (IsSystemOnly && !isSystem)
							continue;

						trades.SafeAdd(security).Add(new Trade
						{
							Id = id,
							Volume = values[volumeIndex].To<decimal>(),
							Price = price,
							Time = dateTime,
							Security = security,
							IsSystem = isSystem,
						});
					}
				}
			}
		}

		private void CombineRtsStd(Dictionary<Security, List<Trade>> trades, DateTime date)
		{
			if (trades == null)
				throw new ArgumentNullException("trades");

			var expiryDate = _exchangeBoard.WorkingTime.GetTPlusNDate(date, 4);

			foreach (var security in trades.Keys.ToList())
			{
				if (security.Type == SecurityTypes.Stock)
				{
					if (security.ExpiryDate == expiryDate)
					{
						var underlying = SecurityStorage.LoadBy("Id", security.UnderlyingSecurityId);

						if (underlying == null ||
							underlying.MinStepSize == 0) // инструмент был создан через FinamHistorySource
						{
							var info = SecurityIdGenerator.GetInfo(security.UnderlyingSecurityId);

							if (underlying == null)
							{
								underlying = new Security
								{
									Id = security.UnderlyingSecurityId,
									Code = info.Item1,
									Name = info.Item1,
									Class = string.Empty,
									ExtensionInfo = new Dictionary<object, object>(),
									ExchangeBoard = ExchangeBoard,
								};
							}

							underlying.Type = security.Type;
							underlying.MinLotSize = security.MinLotSize;
							underlying.MinStepPrice = security.MinStepPrice;
							underlying.MinStepSize = security.MinStepSize;

							SecurityStorage.Save(underlying);
						}
						else
						{
							if (underlying.MinStepSize > security.MinStepSize)
								throw new InvalidOperationException("Инструмент {0} имеет недопустимый шаг цены {1}. Шаг цены базового актива {2} равен {3}.".Put(security.Id, security.MinStepSize, underlying.Id, underlying.MinStepSize));
						}

						var securityTrades = trades[security];

						// TODO
						securityTrades.RemoveWhere(t => t.Price % underlying.MinStepSize != 0);

						trades.SafeAdd(underlying).AddRange(securityTrades.Select(trade => new Trade
						{
							Id = trade.Id,
							Security = underlying,
							Price = trade.Price,
							Time = trade.Time,
							Volume = trade.Volume
						}));
					}
					// TODO:
					//else
					//	throw new InvalidOperationException("Неизвестное для {0} время эскпирации {1}.".Put(security.Id, security.ExpiryDate));

					if (SaveRtsStdCombinedOnly)
					{
						// удаляем РТС-Стандарт вида GAZP18
						trades.Remove(security);
					}
				}
			}
		}
	}
}
